# import os
# import json
#
# def clean_dialogue(dialogue):
#     instruction = dialogue.get("instruction", "").lower()
#     output = dialogue.get("output", "").lower()
#     if any(keyword in instruction or keyword in output for keyword in ["云剪", "剪刀手", "实探","价格","达人","等级",
#                                                                        "IP","ip","地区","米","时间",
#                                                                      "条","钱","要求"]):
#         return dialogue
#     return None
#
# def process_json_file(file_path):
#     with open(file_path, 'r', encoding='utf-8') as file:
#         data = json.load(file)
#
#     cleaned_dialogues = []
#     for entry in data:
#         cleaned_dialogue = clean_dialogue(entry)
#         if cleaned_dialogue:
#             cleaned_dialogues.append(cleaned_dialogue)
#
#     return cleaned_dialogues
#
# def save_to_txt(file_path, dialogues, output_dir):
#     # 确保输出目录存在
#     if not os.path.exists(output_dir):
#         os.makedirs(output_dir)
#
#     output_file_path = os.path.join(output_dir, os.path.basename(file_path).replace('a.json', '.txt'))
#     with open(output_file_path, 'w', encoding='utf-8') as file:
#         for dialogue in dialogues:
#             file.write(f"Instruction: {dialogue['instruction']}\n")
#             file.write(f"Output: {dialogue['output']}\n\n")
#
# def main(json_dir, output_dir):
#     for filename in os.listdir(json_dir):
#         if filename.endswith('a.json'):
#             file_path = os.path.join(json_dir, filename)
#             dialogues = process_json_file(file_path)
#             save_to_txt(file_path, dialogues, output_dir)
#
# if __name__ == "__main__":
#     json_directory = "./save"
#     output_directory = "./save_docs"
#     main(json_directory, output_directory)
#
#
#


import os
import json

def clean_dialogue(dialogue):
    instruction = dialogue.get("instruction", "").lower()
    output = dialogue.get("output", "").lower()
    if any(keyword in instruction or keyword in output for keyword in ["云剪", "剪刀手", "实探", "达人", "等级",
                                                                        "ip", "地区"]):
        return dialogue
    return None

def process_json_file(file_path):
    with open(file_path, 'r', encoding='utf-8') as file:
        data = json.load(file)

    cleaned_dialogues = []
    for entry in data:
        cleaned_dialogue = clean_dialogue(entry)
        if cleaned_dialogue:
            cleaned_dialogues.append(cleaned_dialogue)

    return cleaned_dialogues

def save_to_json(file_path, dialogues, output_dir):
    # 确保输出目录存在
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    if dialogues:  # 只有在有数据的情况下才保存
        output_file_path = os.path.join(output_dir, os.path.basename(file_path))
        with open(output_file_path, 'w', encoding='utf-8') as file:
            json.dump(dialogues, file, ensure_ascii=False, indent=4)

def main(json_dir, output_dir):
    for filename in os.listdir(json_dir):
        if filename.endswith('a.json'):
            file_path = os.path.join(json_dir, filename)
            dialogues = process_json_file(file_path)
            save_to_json(file_path, dialogues, output_dir)

if __name__ == "__main__":
    json_directory = "./save"
    output_directory = "./save_docs"
    main(json_directory, output_directory)
